# -*- coding: utf-8 -*-
import scrapy
from zhipin.items import ZhipinItem


class ZhipinSpiderSpider(scrapy.Spider):
    name = 'zhipin_spider'
    allowed_domains = ['zhipin.com']
    start_urls = ['https://www.zhipin.com/c100010000/e_102/?query=java&page=10']

    def parse(self, response):

        for i in range(1, 31):
            item = ZhipinItem()

            try:
                item['job_name'] = response.xpath(
                    '//*[@id="main"]/div/div[2]/ul/li[' + str(
                        i) + ']/div/div[1]/div[1]/a/div[1]/span[1]/text()').extract()[0]
            except Exception as e:
                print(e)
                break

            try:
                address = response.xpath('//*[@id="main"]/div/div[2]/ul/li[' + str(
                    i) + ']/div/div[1]/div[1]/a/div[1]/span[2]/span/text()').extract()[0]
                item['address'] = address
                if address.find('·') >= 0:
                    item['city'] = address.split('·')[0]
                    item['job_area'] = address.split('·')[1]
                else:
                    item['city'] = address
                    item['job_area'] = address
            except Exception as e:
                print(e)
                item['address'] = 'NULL'
                item['city'] = 'NULL'
                item['job_area'] = 'NULL'

            try:
                salary = response.xpath(
                    '//*[@id="main"]/div/div[2]/ul/li[' + str(
                        i) + ']/div/div[1]/div[1]/a/div[2]/span/text()').extract()[0]
                item['salary'] = salary
                item['min_salary'] = int(salary.split('·')[0].split('-')[0], 10)
                item['max_salary'] = int(salary.split('·')[0].split('-')[1][0:-1], 10)
            except Exception as e:
                print(e)
                item['salary'] = 'NULL'
                item['min_salary'] = int(0)
                item['max_salary'] = int(1)

            try:
                item['education'] = response.xpath(
                    '//*[@id="main"]/div/div[2]/ul/li[' + str(
                        i) + ']/div/div[1]/div[1]/a/div[2]/p/text()[2]').extract()[0]
            except Exception as e:
                print(e)
                item['education'] = 'NULL'

            try:
                item['experience'] = response.xpath(
                    '//*[@id="main"]/div/div[2]/ul/li[' + str(
                        i) + ']/div/div[1]/div[1]/a/div[2]/p/text()[1]').extract()[0]
            except Exception as e:
                print(e)
                item['experience'] = 'NULL'

            try:
                item['company_name'] = response.xpath(
                    '//*[@id="main"]/div/div[2]/ul/li[' + str(i) + ']/div/div[1]/div[2]/div/h3/a/text()').extract()[0]
            except Exception as e:
                print(e)
                break

            try:
                item['url'] = 'www.' + self.allowed_domains[0] + response.xpath(
                    '//*[@id="main"]/div/div[2]/ul/li[' + str(i) + ']/div/div[1]/div[1]/a/@href').extract()[0]
            except Exception as e:
                print(e)
                item['url'] = '没有该职位的url|' + item['job_name'] + '|' + item['company_name']

            try:
                item['industry'] = response.xpath(
                    '//*[@id="main"]/div/div[2]/ul/li[' + str(i) + ']/div/div[1]/div[2]/div/p/text()').extract()[0]
            except Exception as e:
                print(e)
                item['industry'] = 'NULL'

            # item['sel_stage'] = response.xpath(
            #     '//*[@id="main"]/div/div[2]/ul/li[' + str(i) + ']/div/div[1]/div[2]/div/p/text()[2]').extract()[0]

            try:
                item['sel_scale'] = response.xpath(
                    '//*[@id="main"]/div/div[2]/ul/li[' + str(i) + ']/div/div[1]/div[2]/div/p/text()').extract()[-1]
            except Exception as e:
                print(e)
                item['sel_scale'] = 'NULL'

            try:
                item['skill'] = response.xpath(
                    '//*[@id="main"]/div/div[2]/ul/li[' + str(i) + ']/div/div[2]/div[1]/span/text()').extract()
            except Exception as e:
                print(e)
                item['skill'] = 'NULL'

            try:
                item['description'] = (response.xpath('normalize-space(//*[@id="main"]/div/div[2]/ul/li[' + str(
                    i) + ']/div/div[1]/div[1]/a/div[3]/div[2]/div[2])').extract()[0]).strip()
            except Exception as e:
                print(e)
                item['description'] = 'NULL'

            try:
                item['welfare'] = response.xpath(
                    '//*[@id="main"]/div/div[2]/ul/li[' + str(i) + ']/div/div[2]/div[2]/text()').extract()[0]
            except Exception as e:
                print(e)
                item['welfare'] = 'NULL'

            yield item